from model.speed_limit_detector import SpeedLimitDetector
from defense.hgd_trainer import get_HGD_model
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import torch
from glob import glob
import cv2
from tqdm.auto import tqdm
device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
device
batch_size = 2
detector = SpeedLimitDetector(device)
hgd = get_HGD_model(device)
img_paths = glob('../model/datasets/tsinghua_gtsdb_speedlimit/test2017/*.jpg')[:]
preprocessed_imgs = []
for img_path in img_paths:
img = cv2.imread(img_path)
img = detector.preprocess(img)
preprocessed_imgs.append(img)
print(len(preprocessed_imgs))
782
def plot_images(images, title="",transpose=False, rgb=True):
fig, axs = plt.subplots(nrows=2, ncols=2, figsize=(8, 8))
fig.suptitle(title, fontsize=16)
axs = axs.flatten()
for i, img in enumerate(images):
img = img.transpose((1, 2, 0)) if transpose else img
img = np.asarray(img, dtype=np.uint8)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) if rgb else img
axs[i].imshow(img)
axs[i].axis('off')
plt.show()
@torch.no_grad()
def denoise(images):
images_tensor = torch.from_numpy(np.asarray(images)).to(device)
hgd.eval()
noise = hgd(images_tensor)
denoised = images_tensor - noise
return denoised.cpu().numpy(), noise.cpu().numpy()
@torch.no_grad()
def detect_sign(images):
images_tensor = torch.from_numpy(np.asarray(images)).to(device)
outputs = detector.get_model_output(images_tensor)
decoded_outputs = []
for output in outputs:
decoded_outputs.append(detector.decode_model_output(output))
return decoded_outputs
@torch.no_grad()
def get_iou(box1, box2):
# xmin, ymin, xmax, ymax = box
x11, y11, x21, y21 = box1
x12, y12, x22, y22 = box2
yi1 = max(y11, y12)
xi1 = max(x11, x12)
yi2 = min(y21, y22)
xi2 = min(x21, x22)
inter_area = max(((xi2 - xi1) * (yi2 - yi1)), 0)
# Calculate the Union area by using Formula: Union(A,B) = A + B - Inter(A,B)
box1_area = (x21 - x11) * (y21 - y11)
box2_area = (x22 - x12) * (y22 - y12)
union_area = box1_area + box2_area - inter_area
# compute the IoU
iou = inter_area / union_area
return iou
@torch.no_grad()
def calculate_metric(results_to_be_measured,results_name:str):
results = []
conf_threshold = 0.8
IOU_threshold = 0.7
avg_conf = 0
conf_count = 0
for i, result in enumerate(results_to_be_measured):
if result is not None:
(clss, confs, bbs) = result[0],result[1],result[2]
elif benign_images_results[i] is None:
results.append(1)
continue
else:
results.append(0)
continue
for cls, conf, bb in zip(clss,confs,bbs):
if conf < conf_threshold:
continue
avg_conf += conf
conf_count += 1
if benign_images_results[i] is None:
results.append(0)
continue
if benign_images_results[i][2].shape[0] > 1:
max_iou = 0
for bb_idx, bb_benign in enumerate(benign_images_results[i][2]):
IOU = get_iou(bb,bb_benign)
if IOU > max_iou:
max_iou = IOU
idx = bb_idx
else:
max_iou = get_iou(bb,benign_images_results[i][2][0])
idx = 0
if max_iou < IOU_threshold:
results.append(0)
continue
if cls == benign_images_results[i][0][idx]:
results.append(1)
else:
results.append(0)
print(f"{np.asarray(results).mean():.4f} of bounding boxes in {results_name} are correct\nAverage confidence of non-rejected bounding boxes is {avg_conf/conf_count}")
from attack.fgsm import FGSM
from attack.iterative_fgsm import ItFGSM
from attack.pgd import PGD
from model.custom_yolo import yolox_loss, yolox_target_generator
fgsm = FGSM()
fgsm.model = detector.model
fgsm.target_generator = yolox_target_generator
fgsm.loss = yolox_loss
it_fgsm = ItFGSM()
it_fgsm.model = detector.model
it_fgsm.target_generator = yolox_target_generator
it_fgsm.loss = yolox_loss
pgd = PGD()
pgd.model = detector.model
pgd.target_generator = yolox_target_generator
pgd.loss = yolox_loss
fgsm_attacked_images = []
it_fgsm_attacked_images = []
pgd_attacked_images = []
for img in tqdm(preprocessed_imgs):
img = torch.from_numpy(img[None, :, :, :]).to(device)
# fgsm_attacked = fgsm.generate_attack(img, return_numpy=True)[0]
it_fgsm_attacked = it_fgsm.generate_attack(img, return_numpy=True)[0]
# pgd_attacked = pgd.generate_attack(img, return_numpy=True)[0]
# fgsm_attacked_images.append(fgsm_attacked)
it_fgsm_attacked_images.append(it_fgsm_attacked)
#pgd_attacked_images.append(pgd_attacked)
0%| | 0/782 [00:00<?, ?it/s]
with torch.no_grad():
fig, axs = plt.subplots(nrows=2, ncols=2, figsize=(8, 8))
fig.suptitle("Benign Images", fontsize=16)
axs = axs.flatten()
for i, img in enumerate(preprocessed_imgs[:4]):
img = img.transpose((1, 2, 0))
img = np.asarray(img, dtype=np.uint8)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
axs[i].imshow(img)
axs[i].axis('off')
plt.show()
benign_images_results = []
for start_idx in tqdm(range(0,len(preprocessed_imgs),batch_size)):
benign_images_results.append(detect_sign(preprocessed_imgs[start_idx:start_idx+batch_size]))
benign_images_results = np.array(benign_images_results)
benign_images_results = benign_images_results.reshape(benign_images_results.shape[0]*benign_images_results.shape[1])
denoised_benign_images = []
for img in tqdm(preprocessed_imgs):
denoised_benign_images.append(denoise([img])[0])
fig, axs = plt.subplots(nrows=2, ncols=2, figsize=(8, 8))
fig.suptitle("Denoised Benign Images", fontsize=16)
axs = axs.flatten()
for i, img in enumerate(denoised_benign_images[:4]):
img = img[0].transpose((1, 2, 0))
img = np.asarray(img, dtype=np.uint8)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
axs[i].imshow(img)
axs[i].axis('off')
plt.show()
# for img in denoised_benign_images:
# print(detect_sign(img))
0%| | 0/391 [00:00<?, ?it/s]
C:\Users\PC\anaconda3\envs\carlaEnv\lib\site-packages\ipykernel_launcher.py:17: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray. app.launch_new_instance()
0%| | 0/782 [00:00<?, ?it/s]
# plot_images(fgsm_attacked_images[:4], transpose=True, title="FGSM attacked Images")
# detect_sign(fgsm_attacked_images[:4])
denoised_fgsm = []
noise_fgsm = []
with torch.no_grad():
for start_idx in tqdm(range(0,len(fgsm_attacked_images),batch_size)):
results_tuple = denoise(fgsm_attacked_images[start_idx:start_idx+batch_size])
denoised_fgsm.append(results_tuple[0])
noise_fgsm.append(results_tuple[1])
denoised_fgsm = np.array(denoised_fgsm)
denoised_fgsm = denoised_fgsm.reshape(denoised_fgsm.shape[0]*denoised_fgsm.shape[1],denoised_fgsm.shape[2],denoised_fgsm.shape[3],denoised_fgsm.shape[4])
noise_fgsm = np.array(noise_fgsm)
noise_fgsm = noise_fgsm.reshape(noise_fgsm.shape[0]*noise_fgsm.shape[1],noise_fgsm.shape[2],noise_fgsm.shape[3],noise_fgsm.shape[4])
plot_images(noise_fgsm[:4], title="Noise",transpose=True, rgb = True)
plot_images(denoised_fgsm[:4], title="Denoised Images",transpose=True, rgb=True)
denoised_fgsm_results = []
for start_idx in tqdm(range(0,len(denoised_fgsm),batch_size)):
denoised_fgsm_results.append(detect_sign(denoised_fgsm[start_idx:start_idx+batch_size]))
denoised_fgsm_results = np.array(denoised_fgsm_results)
denoised_fgsm_results = denoised_fgsm_results.reshape(denoised_fgsm_results.shape[0]*denoised_fgsm_results.shape[1])
for i,element in enumerate(tqdm(denoised_fgsm_results)):
if element is None:
continue
denoised_fgsm_results[i] = list(element)
0%| | 0/391 [00:00<?, ?it/s]
0%| | 0/391 [00:00<?, ?it/s]
C:\Users\PC\anaconda3\envs\carlaEnv\lib\site-packages\ipykernel_launcher.py:24: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.
0%| | 0/782 [00:00<?, ?it/s]
plot_images(it_fgsm_attacked_images[:4], transpose=True,title = "IT-FGSM Images")
denoised_it_fgsm = []
noise_it_fgsm = []
with torch.no_grad():
for start_idx in tqdm(range(0,len(it_fgsm_attacked_images),batch_size)):
results_tuple = denoise(it_fgsm_attacked_images[start_idx:start_idx+batch_size])
denoised_it_fgsm.append(results_tuple[0])
noise_it_fgsm.append(results_tuple[1])
denoised_it_fgsm = np.array(denoised_it_fgsm)
denoised_it_fgsm = denoised_it_fgsm.reshape(denoised_it_fgsm.shape[0]*denoised_it_fgsm.shape[1],denoised_it_fgsm.shape[2],denoised_it_fgsm.shape[3],denoised_it_fgsm.shape[4])
noise_it_fgsm = np.array(noise_it_fgsm)
noise_it_fgsm = noise_it_fgsm.reshape(noise_it_fgsm.shape[0]*noise_it_fgsm.shape[1],noise_it_fgsm.shape[2],noise_it_fgsm.shape[3],noise_it_fgsm.shape[4])
plot_images(noise_it_fgsm[:4], title="Noise",transpose=True, rgb = True)
plot_images(denoised_it_fgsm[:4], title="Denoised Images",transpose=True, rgb=True)
denoised_it_fgsm_results = []
for start_idx in tqdm(range(0,len(denoised_it_fgsm),batch_size)):
denoised_it_fgsm_results.append(detect_sign(denoised_it_fgsm[start_idx:start_idx+batch_size]))
denoised_it_fgsm_results = np.array(denoised_it_fgsm_results)
denoised_it_fgsm_results = denoised_it_fgsm_results.reshape(denoised_it_fgsm_results.shape[0]*denoised_it_fgsm_results.shape[1])
for i,element in enumerate(tqdm(denoised_it_fgsm_results)):
if element is None:
continue
denoised_it_fgsm_results[i] = list(element)
0%| | 0/391 [00:00<?, ?it/s]
0%| | 0/391 [00:00<?, ?it/s]
C:\Users\PC\anaconda3\envs\carlaEnv\lib\site-packages\ipykernel_launcher.py:24: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.
0%| | 0/782 [00:00<?, ?it/s]
denoised, noise = denoise(it_fgsm_attacked_images)
plot_images(noise[:4], title="Noise",transpose=True)
plot_images(denoised[:4], title="Denoised Images",transpose=True)
denoised_it_fgsm_results = detect_sign(denoised)
with torch.no_grad():
denoised_benign_images = np.array(denoised_benign_images)
denoised_benign_images = denoised_benign_images.reshape(denoised_benign_images.shape[0]*denoised_benign_images.shape[1],denoised_benign_images.shape[2],denoised_benign_images.shape[3],denoised_benign_images.shape[4])
denoised_benign_results = []
for start_idx in tqdm(range(0,len(denoised_benign_images),batch_size)):
denoised_benign_results.append(detect_sign(denoised_benign_images[start_idx:start_idx+batch_size]))
denoised_benign_results = np.array(denoised_benign_results)
denoised_benign_results = denoised_benign_results.reshape(denoised_benign_results.shape[0]*denoised_benign_results.shape[1])
for i,element in enumerate(tqdm(denoised_benign_results)):
if element is None:
continue
denoised_benign_results[i] = list(element)
0%| | 0/391 [00:00<?, ?it/s]
C:\Users\PC\anaconda3\envs\carlaEnv\lib\site-packages\ipykernel_launcher.py:8: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.
0%| | 0/782 [00:00<?, ?it/s]
with torch.no_grad():
fgsm_results = []
for start_idx in tqdm(range(0,len(fgsm_attacked_images),batch_size)):
fgsm_results.append(detect_sign(fgsm_attacked_images[start_idx:start_idx+batch_size]))
fgsm_results = np.array(fgsm_results)
fgsm_results = fgsm_results.reshape(fgsm_results.shape[0]*fgsm_results.shape[1],fgsm_results.shape[2])
for i,element in enumerate(tqdm(fgsm_results)):
if element is None:
continue
fgsm_results[i] = list(element)
0%| | 0/391 [00:00<?, ?it/s]
C:\Users\PC\anaconda3\envs\carlaEnv\lib\site-packages\ipykernel_launcher.py:6: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.
0%| | 0/782 [00:00<?, ?it/s]
with torch.no_grad():
it_fgsm_results = []
for start_idx in tqdm(range(0,len(it_fgsm_attacked_images),batch_size)):
it_fgsm_results.append(detect_sign(it_fgsm_attacked_images[start_idx:start_idx+batch_size]))
it_fgsm_results = np.array(it_fgsm_results)
it_fgsm_results = it_fgsm_results.reshape(it_fgsm_results.shape[0]*it_fgsm_results.shape[1],it_fgsm_results.shape[2])
for i,element in enumerate(tqdm(it_fgsm_results)):
if element is None:
continue
it_fgsm_results[i] = list(element)
0%| | 0/391 [00:00<?, ?it/s]
C:\Users\PC\anaconda3\envs\carlaEnv\lib\site-packages\ipykernel_launcher.py:6: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.
0%| | 0/782 [00:00<?, ?it/s]
calculate_metric(denoised_benign_results, "Denoised Benign")
0.9928 of bounding boxes in Denoised Benign are correct Average confidence of non-rejected bounding boxes is 0.8764400858741375
calculate_metric(fgsm_results, "FGSM Images")
0.6065 of bounding boxes in FGSM Images are correct Average confidence of non-rejected bounding boxes is 0.8531521495293688
calculate_metric(denoised_fgsm_results, "denoised FGSM")
0.8147 of bounding boxes in denoised FGSM are correct Average confidence of non-rejected bounding boxes is 0.8663147422832205
calculate_metric(it_fgsm_results,"IT-FGSM Results")
0.0034 of bounding boxes in IT-FGSM Results are correct Average confidence of non-rejected bounding boxes is 0.8726816055745289
calculate_metric(denoised_it_fgsm_results, "denoised IT FGSM")
0.6752 of bounding boxes in denoised IT FGSM are correct Average confidence of non-rejected bounding boxes is 0.862736709256029